package com.chance.cc.crawler.development.scripts.amazon;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;

public class AmazonCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(AmazonCrawlerScript.class);//日志
    private static final String DOMAIN = "amazon";
    private static final String SITE = "commodity";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //    https://www.amazon.cn/s?k=nike
    private static final String HomeRegulars = "https://www.amazon.cn";//主页
    //    https://www.amazon.cn/s?k=nike&page=1
    private static final String NewsRegulars = "https://www.amazon.cn/s\\?k=\\S*&page=\\d*";//文章列表链接
    //    https://www.amazon.cn/dp/B00803ZCOI
    private static final String ContentRegulars = "https://www.amazon.cn/dp/\\S*";//文章详情链接
    //    https://www.amazon.cn/product-reviews/B07H7SF5TH?pageNumber=1
    private static final String CommentRegulars = "https://www.amazon.cn/product-reviews/\\S*pageNumber=\\d*";//评论下一页

    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(NewsRegulars);
        addUrlRegular(ContentRegulars);
        addUrlRegular(CommentRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(NewsRegulars)) {
            newsRequest(crawlerRecord, page, parsedLinks);
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {
            articleRequest(crawlerRecord, page, parsedLinks);
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(CommentRegulars)) {
            commentRequest(crawlerRecord, page, parsedLinks);
        }
        return parsedLinks;
    }

    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
            String url = "https://www.amazon.cn/s?k=" + name.get(i) + "&page=1";
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(record);
        }

    }

    private void newsRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String nextStr = page.getHtml().xpath("//a[./text()=\"下一页\"]/@href").get();
        String url = page.getRequest().getUrl();
        if (url.matches(NewsRegulars) && nextStr != null) {
            String[] split = url.replaceAll("s\\?k=", "").replaceAll("&page=", "/").split("/");
            String nextPageUrl = "https://www.amazon.cn/s?k=" + split[split.length - 2] + "&page=" + (Integer.parseInt(split[split.length - 1]) + 1);
            CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .needParsed(true)
                    .httpHead("Connection", "keep-alive")
                    .httpHead("Cache-Control", "max-age=0")
                    .httpHead("rtt", "100")
                    .httpHead("downlink", "10")
                    .httpHead("ect", "4g")
                    .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                    .httpHead("sec-ch-ua-mobile", "?0")
                    .httpHead("Upgrade-Insecure-Requests", "1")
                    .httpHead("User-Agent", getRandomUA())
                    .httpHead("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
                    .httpHead("Sec-Fetch-Site", "same-origin")
                    .httpHead("Sec-Fetch-User", "?1")
                    .httpHead("Sec-Fetch-Dest", "document")
                    .httpHead("Accept-Language", "zh-CN,zh;q=0.9")
                    .httpHead("Cookie", "session-id=462-8168812-7024355; i18n-prefs=CNY; lc-acbcn=zh_CN; ubid-acbcn=460-7097111-9030419; session-token=ugupkmxReZDKiKU9dL2bTGGOPx7wT4YHSOqf8ABSKwOEdq0RIJRHKMf4phDM7NatJN8aKJR16euN4hvm/azrL/N0PpnggNhDcOs51w65DOCWe2SNI9Li/NwcPUe6lzCxB6U76vQRSCWSWGtFaPSqeUiBGWOfULVJ6Hi0qYFfXpVn61b1Ne6k4q8Ez/wi4uy7; session-id-time=2082787201l; csm-hit=tb:X1YHY8ZT6ANH2MPSB6AF+s-HHNT7Y9CTWD0RJF70XG5|1634023110426&t:1634023110427&adb:adblk_no")
                    .build();
            parsedLinks.add(turnPageRequest);
        }

        //        明细页解析
        List<Selectable> nodes = page.getHtml().xpath("//div[@class=\"sg-col-4-of-12 s-result-item s-asin sg-col-4-of-16 sg-col s-widget-spacing-small sg-col-4-of-20\"]").nodes();//所有文章
        for (Selectable node : nodes) {
            String urlId = node.xpath(".//a[@class=\"a-link-normal a-text-normal\"]/@href").get();
            if (StringUtils.isBlank(urlId)) {
                continue;
            }
            String[] split = urlId.split("/");
            String articleUrl = "https://www.amazon.cn/dp/" + split[2];
            String price = node.xpath(".//span[@class=\"a-offscreen\"]/text()").get();
            crawlerRecord.tagsCreator().bizTags().addCustomKV("price", price);

            CrawlerRequestRecord articleRecord = CrawlerRequestRecord.builder()//解析文章
                    .itemPageRequest(crawlerRecord)
                    .recordKey(articleUrl)
                    .httpUrl(articleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .needWashed(true)
                    .httpHead("Connection", "keep-alive")
                    .httpHead("Cache-Control", "max-age=0")
                    .httpHead("rtt", "100")
                    .httpHead("downlink", "7.3")
                    .httpHead("ect", "4g")
                    .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                    .httpHead("sec-ch-ua-mobile", "?0")
                    .httpHead("Upgrade-Insecure-Requests", "1")
                    .httpHead("User-Agent", getRandomUA())
                    .httpHead("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
                    .httpHead("Sec-Fetch-Site", "same-origin")
                    .httpHead("Sec-Fetch-User", "?1")
                    .httpHead("Sec-Fetch-Dest", "document")
                    .httpHead("Accept-Language", "zh-CN,zh;q=0.9")
                    .httpHead("Cookie", "session-id=462-8168812-7024355; i18n-prefs=CNY; lc-acbcn=zh_CN; ubid-acbcn=460-7097111-9030419; session-token=EHZ5kqcTT7y8ruFHQmzkrr0E9KGqXxs+N4bHe/1I3Nr+7tritzLoD5zKNVg8T6QVIRGbxNrvQ40TPmfpb/myUl+cTe9qvOiP+Ma9vFQc3gj5UM3opnsh07ChZl830KuqdZ/wy0TMtM8xR/3lEZK+72IVllw7PoIAZFKb8yBjbCi3pqLkY9uBDYk0hcgFt1qQ; ")
                    .build();
            parsedLinks.add(articleRecord);

//            String commentUrl = "https://www.amazon.cn/product-reviews/" + split[2] + "?pageNumber=1";
//            crawlerRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls,articleUrl);
//            CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()//解析文章
//                    .itemPageRequest(crawlerRecord)
//                    .recordKey(commentUrl)
//                    .httpUrl(commentUrl)
//                    .releaseTime(System.currentTimeMillis())
//                    .copyBizTags()
//                    .copyResultTags()
//                    .needWashed(true)
//                    .httpHead("Connection","keep-alive")
//                    .httpHead("Cache-Control","max-age=0")
//                    .httpHead("rtt","100")
//                    .httpHead("downlink","7.3")
//                    .httpHead("ect","4g")
//                    .httpHead("sec-ch-ua","\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
//                    .httpHead("sec-ch-ua-mobile","?0")
//                    .httpHead("Upgrade-Insecure-Requests","1")
//                    .httpHead("User-Agent",getRandomUA())
//                    .httpHead("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
//                    .httpHead("Sec-Fetch-Site","same-origin")
//                    .httpHead("Sec-Fetch-User","?1")
//                    .httpHead("Sec-Fetch-Dest","document")
//                    .httpHead("Accept-Language","zh-CN,zh;q=0.9")
//                    .httpHead("Cookie","session-id=462-8168812-7024355; i18n-prefs=CNY; lc-acbcn=zh_CN; ubid-acbcn=460-7097111-9030419; session-token=EHZ5kqcTT7y8ruFHQmzkrr0E9KGqXxs+N4bHe/1I3Nr+7tritzLoD5zKNVg8T6QVIRGbxNrvQ40TPmfpb/myUl+cTe9qvOiP+Ma9vFQc3gj5UM3opnsh07ChZl830KuqdZ/wy0TMtM8xR/3lEZK+72IVllw7PoIAZFKb8yBjbCi3pqLkY9uBDYk0hcgFt1qQ; ")
//                    .build();
//            parsedLinks.add(commentRecord);
        }
    }

    private void articleRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls, page.getRequest().getUrl());
        String[] split = page.getRequest().getUrl().split("/");
        String nextPageUrl = "https://www.amazon.cn/product-reviews/" + split[split.length - 1] + "?pageNumber=1";

        KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
        CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
        CrawlerRequestRecord articleRecord = CrawlerRequestRecord.builder()//解析文章
                .itemPageRequest(crawlerRecord)
                .recordKey(nextPageUrl)
                .httpUrl(nextPageUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .copyResultTags()
                .needWashed(true)
                .notFilterRecord()
                .httpHead("Connection", "keep-alive")
                .httpHead("Cache-Control", "max-age=0")
                .httpHead("rtt", "100")
                .httpHead("downlink", "7.3")
                .httpHead("ect", "4g")
                .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                .httpHead("sec-ch-ua-mobile", "?0")
                .httpHead("Upgrade-Insecure-Requests", "1")
                .httpHead("User-Agent", getRandomUA())
                .httpHead("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
                .httpHead("Sec-Fetch-Site", "same-origin")
                .httpHead("Sec-Fetch-User", "?1")
                .httpHead("Sec-Fetch-Dest", "document")
                .httpHead("Accept-Language", "zh-CN,zh;q=0.9")
                .httpHead("Cookie", "session-id=462-8168812-7024355; i18n-prefs=CNY; lc-acbcn=zh_CN; ubid-acbcn=460-7097111-9030419; session-token=EHZ5kqcTT7y8ruFHQmzkrr0E9KGqXxs+N4bHe/1I3Nr+7tritzLoD5zKNVg8T6QVIRGbxNrvQ40TPmfpb/myUl+cTe9qvOiP+Ma9vFQc3gj5UM3opnsh07ChZl830KuqdZ/wy0TMtM8xR/3lEZK+72IVllw7PoIAZFKb8yBjbCi3pqLkY9uBDYk0hcgFt1qQ; ")
                .build();
        articleRecord.setFilter(filterInfoRecord.getFilter());
        articleRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
        parsedLinks.add(articleRecord);

    }

    private void commentRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String nextStr = page.getHtml().xpath("//a[./text()=\"下一页\"]/@href").get();
        String url = page.getRequest().getUrl();
        if (url.matches(CommentRegulars) && nextStr != null) {
//            https://www.amazon.cn/product-reviews/B07H7SF5TH?pageNumber=1
            try {
                String[] split = url.replaceAll("\\?pageNumber=", "/").split("/");
                String nextPageUrl = "https://www.amazon.cn/product-reviews/" + split[split.length - 2] + "?pageNumber=" + (Integer.parseInt(split[split.length - 1]) + 1);

                KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
                CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
                CrawlerRequestRecord articleRecord = CrawlerRequestRecord.builder()//解析文章
                        .itemPageRequest(crawlerRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .copyResultTags()
                        .needWashed(true)
                        .notFilterRecord()
                        .httpHead("Connection", "keep-alive")
                        .httpHead("Cache-Control", "max-age=0")
                        .httpHead("rtt", "100")
                        .httpHead("downlink", "7.3")
                        .httpHead("ect", "4g")
                        .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                        .httpHead("sec-ch-ua-mobile", "?0")
                        .httpHead("Upgrade-Insecure-Requests", "1")
                        .httpHead("User-Agent", getRandomUA())
                        .httpHead("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
                        .httpHead("Sec-Fetch-Site", "same-origin")
                        .httpHead("Sec-Fetch-User", "?1")
                        .httpHead("Sec-Fetch-Dest", "document")
                        .httpHead("Accept-Language", "zh-CN,zh;q=0.9")
                        .httpHead("Cookie", "session-id=462-8168812-7024355; i18n-prefs=CNY; lc-acbcn=zh_CN; ubid-acbcn=460-7097111-9030419; session-token=EHZ5kqcTT7y8ruFHQmzkrr0E9KGqXxs+N4bHe/1I3Nr+7tritzLoD5zKNVg8T6QVIRGbxNrvQ40TPmfpb/myUl+cTe9qvOiP+Ma9vFQc3gj5UM3opnsh07ChZl830KuqdZ/wy0TMtM8xR/3lEZK+72IVllw7PoIAZFKb8yBjbCi3pqLkY9uBDYk0hcgFt1qQ; ")
                        .build();
                articleRecord.setFilter(filterInfoRecord.getFilter());
                articleRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
                parsedLinks.add(articleRecord);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {//文章互动量
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {//文章链接
                crawlerDataList.addAll(articleListWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {//文章链接
                crawlerDataList.addAll(articleWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(CommentRegulars)) {//文章互动量
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(comment)) {//文章链接
                crawlerDataList.addAll(commentWash(crawlerRecord, page));
            }
        }

        return crawlerDataList;
    }

    //清洗文章链接
    public List<CrawlerData> articleListWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
//        log.info("清洗文章链接");
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            String articleUrl = page.getRequest().getUrl();
            String[] articleId = articleUrl.split("/");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, article.enumVal(), articleId[articleId.length - 1]))
                    .resultLabelTag(article)
                    .url(articleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV("releaseTimeToLong", System.currentTimeMillis() + "")
                    .addContentKV("articleUrl", articleUrl)
                    .addContentKV(AICCommonField.Tag_Site_Info, "商品")
                    .flowInPipelineTag("redis")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }


    //解析文章
    public List<CrawlerData> articleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("解析文章");
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            String[] split1 = page.getRawText().split("'initial'");
            if (split1.length < 2) {
                return crawlerArticleDataList;
            }
            String json = split1[1].split("'colorToAsin")[0];
            json = json.trim().substring(1, json.length() - 3);
            String articleUrl = page.getRequest().getUrl();//文章url
            String title = page.getHtml().xpath("//span[@id=\"productTitle\"]/text()").get();//标题
            title = title.replaceAll("\\\n", "");
            String price = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("price");
            if (price == null) {
                price = "0";
            }
            price = price.replaceAll("&amp;yen;", "￥");
            List<String> picture = new Json(json).jsonPath($_type + "..large").all();//图片

            StringBuffer pictures = new StringBuffer();//拼接图片
            for (String text : picture) {
                pictures.append(text).append("\0x1");
            }
            String[] split = articleUrl.replaceAll("goodsId=", "/").trim().split("/");//关键字
            String articleId = split[split.length - 1];
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, article.enumVal(), articleId))
                    .resultLabelTag(article)
                    .url(articleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV(AICCommonField.Field_Title, title)
                    .addContentKV(AICCommonField.Field_Produce_Price, price)
                    .addContentKV(AICCommonField.Field_Images, pictures.toString())
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }


    //  评论清洗
    private Collection<? extends CrawlerData> commentWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Selectable> nodes = page.getHtml().xpath("//div[@class=\"a-section review aok-relative\"]").nodes();
        for (Selectable node : nodes) {
            log.info("评论清洗");
            try {
                String title = node.xpath(".//a[@class=\"a-size-base a-link-normal review-title a-color-base review-title-content a-text-bold\"]//span/text()").get();//评论标题
                String content = node.xpath(".//span[@class=\"a-size-base review-text review-text-content\"]//span/text()").get();//评论内容
                String releaseTime = node.xpath(".//span[@class=\"a-size-base a-color-secondary review-date\"]/text()").get();//回复时间
                String user = node.xpath(".//span[@class=\"a-profile-name\"]/text()").get();//评论用户
                String userId = node.xpath(".//a[@class=\"a-profile\"]/@href").get();//评论用户id
                String[] userIdStr = userId.replaceAll("account.", "/").split("/");
                userId = userIdStr[userIdStr.length - 2];
                String commentId = node.xpath("./@id").get();//评论id

                List<String> picture = page.getHtml().xpath("//div[@class=\"review-image-tile-section\"]//img/@src").all();//图片
                StringBuffer pictures = new StringBuffer();//拼接图片
                for (String text : picture) {
                    pictures.append(text).append("\0x1");
                }

                String articleUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url
                String[] split = articleUrl.split("/");//关键字
                String articleId = split[split.length - 1];
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, comment.enumVal(), commentId))
                        .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, article.enumVal(), articleId))
                        .resultLabelTag(comment)
                        .url(articleUrl)
                        .releaseTime(DateUtils.parseDate(releaseTime, "yyyy年MM月dd日").getTime())
                        .addContentKV(AICCommonField.Field_Title, title)
                        .addContentKV(AICCommonField.Field_Content, content)
                        .addContentKV(AICCommonField.Field_Author, user)
                        .addContentKV(AICCommonField.Field_Author_Id, userId)
                        .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
//                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
//                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 50) {
                log.error("autohome download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }


    private static List<String> agentList = new ArrayList<>();

    static {
        agentList.add("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36");
        agentList.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko");
        agentList.add("Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)");
        agentList.add("Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)");
        agentList.add("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/532.2");
        agentList.add("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.28.3 (KHTML, like Gecko) Version/3.2.3 ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/525.28.3");
        agentList.add("Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16");
        agentList.add("Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14");
        agentList.add("Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14");
        agentList.add("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14");
        agentList.add("Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02");
        agentList.add("Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00");
        agentList.add("Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00");
        agentList.add("Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00");
        agentList.add("Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1");
        agentList.add("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0");
        agentList.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0");
        agentList.add("Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.13 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3756.400 QQBrowser/10.5.4043.400");
    }

    private static String getRandomUA() {
        return agentList.get(RandomUtils.nextInt(0, agentList.size() - 1));
    }

}
